import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


/**
 * Jsoup 实现爬虫的例子
 * 功能：北京顺义区 中标信息、招标信息 获取
 *
 * @author zhangjian
 * @version 1.0.0
 * @since 2022/01/06 16:25
 */
@Slf4j
public class Spider {
  public static void main(String[] args) throws Exception {
    Spider spider = new Spider();
    // 中标信息打印
    List<String> win = spider.spiderWinner();
    for (String s : win) {
      System.out.println(s);
    }

    // 招标信息打印
    /*ArrayList<String> find = spider.spiderFind();
    for (String s : find) {
      System.out.println(s);
    }*/


  }

  /**
   * 招标信息
   * http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbgg/index.html
   * @return
   */
  public ArrayList<String> spiderFind(){
    String urlPre = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbgg/";
    ArrayList<String> text = new ArrayList<>();

    // 获取到所有要下载的 URL
    Map<String, String> map = getFindUrl();
    for (Map.Entry<String, String> entry : map.entrySet()) {
      String fullUrl = urlPre + entry.getKey().substring(2);
       String one = getOneFind(fullUrl);
       text.add(one + "," + fullUrl);
    }
    return text;
  }

  @SneakyThrows
  private String getOneFind(String fullUrl) {
    String html = Jsoup.connect(fullUrl).timeout(10000).get().html();
    Document document = Jsoup.parse(html);
    Element body = document.body();

    String title = "body > div:nth-child(2) > div:nth-child(2) > span";

    // 项目概况
    String summery = "body > div:nth-child(2) > div:nth-child(3) > div > blockquote > p:nth-child(2)";
    String summery2 = "body > div:nth-child(2) > div:nth-child(3) > blockquote";
    // 最高限价
    String maxMoney = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(7)";
    String maxMoney2 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(8)";

    String maxMoney3 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(7)";

    String projectNo = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(12)";
    String projectNo2 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(5)";
    String projectNo3 = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(3)";

    String projectName = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(6)";
    String projectName2 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(7)";
    String projectName3 = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(4)";

    String summeryEle = body.select(summery).text();
    String summeryEle2 = body.select(summery2).text();
    String maxMoneyEle = body.select(maxMoney).text();
    String maxMoneyEle2 = body.select(maxMoney2).text();
    String maxMoneyEle3 = body.select(maxMoney3).text();

    String titleEle = body.select(title).text();

    String projectNoEle = body.select(projectNo).text();
    String projectNoEle2 = body.select(projectNo2).text();
    String projectNoEle3 = body.select(projectNo3).text();

    String projectNameEle = body.select(projectName).text();
    String projectNameEle2 = body.select(projectName2).text();
    String projectNameEle3 = body.select(projectName3).text();


    String finalSummery = StringUtils.isNotEmpty(summeryEle) ? summeryEle : summeryEle2;
    String dt = finalSummery.replaceAll(".*([\\d]{4}[-|年][\\d]{2}[-|月][\\d]{2}[日]? [\\d]{2}:[\\d]{2}（北京时间）).*", "$1");

    return new StringBuilder(titleEle).append(",")
        .append(finalSummery).append(",")
        .append("采购需求：".equals(stringNvl(maxMoneyEle, maxMoneyEle2)) ? maxMoneyEle3 : stringNvl(maxMoneyEle, maxMoneyEle2)).append(",")
        .append(NvlAndContainsKey("项目编号", projectNoEle, projectNoEle2, projectNoEle3)).append(",")
        .append(stringNvl(projectNameEle, projectNameEle2, projectNameEle3)).append(",")
        .append(dt)
        .toString();

  }

  private String NvlAndContainsKey(String key, String... text) {
    for (String s : text) {
      if (StringUtils.isNotEmpty(s) && s.contains(key)) {
        return s;
      }
    }
    return "";
  }

  private String stringNvl(String... text) {
    for (String s : text) {
      if (StringUtils.isNotEmpty(s)) {
        return s;
      }
    }
    return "";
  }

  @SneakyThrows
  private Map<String, String> getFindUrl() {
    ArrayList<String> urls = new ArrayList<>();

    String urlTemplate = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbgg/index_%d.html";
    String zero = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbgg/index.html";
    urls.add(zero);

    for (int i = 1; i <=142; i++) {
      urls.add(String.format(urlTemplate, i));
    }


    HashMap<String, String> result = new HashMap<>();
    for (String url : urls) {
      log.info("get url={}", url);
      String html = Jsoup.connect(url).get().html();
      Document document = Jsoup.parse(html);
      Element body = document.body();
      Elements elements = body.children().get(0).children();
      for (Element element : elements) {
        String href = element.select("a").attr("href");
        String title = element.select("a").text();
        if(title.contains("顺义")){
          result.put(href, title);
        }
      }
    }
    return result;
  }

  /**
   * 中标信息
   * @return 抓取到的所有中标信息内容
   */
  public List<String> spiderWinner(){
    String urlPre = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbjggg/";

    ArrayList<String> text = new ArrayList<>();

    Map<String, String> map = getUrl();
    for (Map.Entry<String, String> entry : map.entrySet()) {
      String fullUrl = urlPre + entry.getKey().substring(2);
      String one = getOne(fullUrl);
      text.add(one + "," + fullUrl);
    }

    return text;
  }

  /**
   * 请求主网页、获取子网页的地址列表和名称
   *
   * @return 子网页的地址列表和名称
   */
  @SneakyThrows
  public Map<String, String> getUrl(){
    ArrayList<String> urls = new ArrayList<>();
    String urlTemplate = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbjggg/index_%s.html";
    String zero = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbjggg/index.html";
    urls.add(zero);
    for (int i = 1; i <=142; i++) {
      urls.add(String.format(urlTemplate, i));
    }

    HashMap<String, String> result = new HashMap<>();
    for (String url : urls) {
      log.info("get url={}", url);
      String html = Jsoup.connect(url).timeout(60000).get().html();
      Document document = Jsoup.parse(html);
      Element body = document.body();
      Elements elements = body.children().get(0).children();
      for (Element element : elements) {
        String href = element.select("a").attr("href");
        String title = element.select("a").text();
        if(title.contains("顺义")){
          result.put(href, title);
        }
      }
    }
    return result;
  }

  /**
   * 中标信息网页内容提取
   * String url = "http://www.ccgp-beijing.gov.cn/xxgg/qjzfcggg/qjzbjggg/t20220105_1399646.html";
   * 实现 XPath 路劲配置
   *
   * @param url 要获取的详情页 url
   * @return 一行 csv 格式的字符串
   */
  @SneakyThrows
  public String getOne(String url){

    String html = Jsoup.connect(url).get().html();
    Document document = Jsoup.parse(html);
    Element body = document.body();

    String title = "body > div:nth-child(2) > div:nth-child(2) > span";

    String projectNo = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(3) > strong";
    String projectNo2 = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(1) > strong";
    String projectName = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(4) > strong";
    String projectName2 = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(2) > strong";

    // 中标公司名称
    String coopName = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(9)";
    String coopName2 = "body > div:nth-child(2) > div:nth-child(3) > table > tbody > tr:nth-child(2) > td:nth-child(2)";
    String coopName3 = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(7)";

    // 中标金额
    String money = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(4)";
    String money2 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(11)";
    String money3 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(6)";

    // 采购人名称
    String buyName = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(41)";
    String buyName2 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(45)";
    String buyName3 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(38)";
    String buyName4 = "body > div:nth-child(2) > div:nth-child(3) > div > p:nth-child(40)";
    String buyName5 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(43)";
    String buyName6 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(47)";
    String buyName7 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(42)";
    String buyName8 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(49)";
    String buyName9 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(36)";
    String buyName10 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(48)";
    String buyName11 = "body > div:nth-child(2) > div:nth-child(3) > p:nth-child(44)";


    String titleEle = body.select(title).text();
    String projectNoEle = body.select(projectNo).text();
    String projectNoEle2 = body.select(projectNo2).text();
    String projectNameEle = body.select(projectName).text();
    String projectNameEle2 = body.select(projectName2).text();
    String companyNameEle = body.select(coopName).text();
    String companyNameEle2 = body.select(coopName2).text();
    String companyNameEle3 = body.select(coopName3).text();

    String moneyEle = body.select(money).text();
    String moneyEle2 = body.select(money2).text();
    String moneyEle3 = body.select(money3).text();

    String buyNameEle = body.select(buyName).text();
    String buyNameEle2 = body.select(buyName2).text();
    String buyNameEle3 = body.select(buyName3).text();
    String buyNameEle4 = body.select(buyName4).text();
    String buyNameEle5 = body.select(buyName5).text();
    String buyNameEle6 = body.select(buyName6).text();
    String buyNameEle7 = body.select(buyName7).text();
    String buyNameEle8 = body.select(buyName8).text();
    String buyNameEle9 = body.select(buyName9).text();
    String buyNameEle10 = body.select(buyName10).text();
    String buyNameEle11 = body.select(buyName11).text();

    StringBuilder stringBuilder = new StringBuilder();
    return stringBuilder
        .append(titleEle).append(",")
        .append(stringNvl(projectNoEle, projectNoEle2)).append(",")
        .append(stringNvl(projectNameEle, projectNameEle2)).append(",")
        .append(stringNvl(companyNameEle, companyNameEle2, companyNameEle3)).append(",")
        .append(NvlAndContainsKey("金额", moneyEle, moneyEle2, moneyEle3)).append(",")
        .append(NvlAndContainsKey("名 称", buyNameEle, buyNameEle2, buyNameEle3, buyNameEle4, buyNameEle5,
            buyNameEle6, buyNameEle7, buyNameEle8, buyNameEle9, buyNameEle10, buyNameEle11))
        .toString();

  }
}
